﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

namespace scomA3proj
{
    public class Tokenizer_James : ITokenizer_James
    {
        public List<string> Tokenize(string str, List<string> tokenDelims, bool toLower, bool trim, bool removeWhitespace, bool removeDelimTokens)
        {
            List<string> res = new List<string>();
            if (toLower)
            {
                str = str.ToLower();
            }
            string regexExp = "";
            bool first = true;
            if (removeWhitespace)
            {
                regexExp += @"\s+";
                first = false;
            }

            foreach (string s in tokenDelims)
            {
                string temp = Regex.Escape(s);
                if (!first) regexExp += "|";
                first = false;
                regexExp += "(" + temp + ")";
            }
            string[] arr = Regex.Split(str, regexExp);
            if (removeWhitespace)
            {
                arr = arr.Where(item => (!Regex.IsMatch(item, @"\s+") && item.Length != 0)).ToArray();
            }
            else
            {
                arr = arr.Where(item => (item.Length != 0)).ToArray();
            }
            
            if (trim)
            {
                for (int i = 0; i < arr.Length; i++)
                {
                    arr[i] = arr[i].Trim();
                }
            }
            return arr.ToList();
        }
    }
}
